﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using System.Net;
using System.IO;
using System.Web;

namespace NewsDiscoveryEngine
{

    /*
     * 
     * Note: This class is designed to detect the menu items in the website, this will be enhanced to include in the auto job
     * Currently this will be used in the UI project to help user to detect the menu.
     * 
     */

    public class HtmlLinkOld
    {
        public HtmlLinkOld()
        {
            IsFeedType = false;
        }

        public string Link { get; set; }
        public string Title { get; set; }

        public bool IsFeedType { get; set; }
    }

    /// <summary>
    /// This class is used intended to created utility functions to crawl the web site for getting website inforarmation, Website Menus and also RSS information.
    /// 
    /// Input - One of the article Url.
    /// 
    /// Finding the webSite information:-
    /// - Header can give the website.
    /// 
    /// 
    /// Finding the links: -
    /// - From the single url find all the links
    /// - From the links found in the above step will be used to get more pages and find those links as well.
    /// - After getting decent number of the html document for the source domain, We need to take the aggregate count of the links
    /// - So, more hits gets the link, becomes the clue for determining the Menu.
    /// 
    /// Finding the Feed links: - 
    /// 
    /// - From the header tag, we could find,
    /// - From the links, we can search for the some keywords like, RSS and get hint 
    /// - From the crawling process we could be able to tell the content type of the url.
    /// 
    /// 
    /// </summary>
    public class AutoWebSiteDiscoverer
    {

        public AutoWebSiteDiscoverer()
        {

        }

        public List<HtmlLinkOld> GetLinks(HtmlDocument doc,Uri baseUri)
        {
            var nodes = doc.DocumentNode.SelectNodes("//a");

            List<HtmlLinkOld> links = new List<HtmlLinkOld>();
            foreach (var node in nodes)
            {
                if (node.Attributes["href"] == null) continue;

                string link = node.Attributes["href"].Value;
                string title = node.InnerText; //HttpUtility.HtmlDecode(node.InnerText);

                Uri tempUri = new Uri(baseUri, link);


                HtmlLinkOld HtmlLinkOld = new HtmlLinkOld { Link = tempUri.AbsoluteUri, Title = title };

                links.Add(HtmlLinkOld);
            }

            return links;
        }

        public void GetWebSiteData(string url)
        {

            List<HtmlDocument> documnets = new List<HtmlDocument>();

            // Collection Needed...

            // - Open Url collection (contains Duplicate entries)
            // - Url Queue for crawling...
            // - Urls that already crawled.
            // - Syndycations related links



            // Load document 

            // Find All the links that are inside the same domain.

            // Store this document for future usage

            // For each of the above links, download the links

            // put all the links inside collection, take the 

            HtmlDocument doc = new HtmlDocument();

            Uri urlInfo = new Uri(url);

            string mainDomain = urlInfo.DnsSafeHost;

            var uri = new Uri(url);
            var httpWebRequest = (HttpWebRequest)WebRequest.Create(uri);
            httpWebRequest.AllowAutoRedirect = true;
            httpWebRequest.Method = "GET";

            var response = httpWebRequest.GetResponse() as HttpWebResponse;
            string responseText = "";
            using (var reader = new StreamReader(response.GetResponseStream()))
            {
                responseText = reader.ReadToEnd();
            }
            response.Close();
            if (response.StatusCode == HttpStatusCode.OK && !string.IsNullOrEmpty(responseText))
            {
                doc.LoadHtml(responseText);


                documnets.Add(doc);


                var nodes = doc.DocumentNode.SelectNodes("//a");


                Uri temUriForFixingRelativePaths;
                foreach (var node in nodes)
                {
                    if (node.Attributes["href"].Value.ToLower().Contains(mainDomain))
                    {
                        if (!string.IsNullOrEmpty(node.InnerText) && node.InnerText == node.InnerHtml) // images also can be there, so <img> tag will not be same to text.
                        {
                            string link =  node.Attributes["href"].Value;
                            string title = node.InnerText;

                            temUriForFixingRelativePaths = new Uri(urlInfo, link);

                            HtmlLinkOld HtmlLinkOld = new HtmlLinkOld { Link = temUriForFixingRelativePaths.AbsoluteUri, Title = title };


                            HtmlWeb webUtility = new HtmlWeb();
                            documnets.Add(webUtility.Load(HtmlLinkOld.Link));

                            if (documnets.Count > 6)
                            {
                                break;
                            }
                        }
                    }
                    else
                    {

                        if (!string.IsNullOrEmpty(node.InnerText)  && node.InnerText == node.InnerHtml) // images also can be there, so <img> tag will not be same to text.
                        {
                            string link = node.Attributes["href"].Value;
                            string title = node.InnerText;
                            //string title = HttpUtility.HtmlDecode(node.InnerText);


                            temUriForFixingRelativePaths = new Uri(urlInfo, link);

                            HtmlLinkOld HtmlLinkOld = new HtmlLinkOld { Link = temUriForFixingRelativePaths.AbsoluteUri, Title = title };


                            Uri tempLocation = new Uri(HtmlLinkOld.Link);

                            if (tempLocation.DnsSafeHost.Contains(mainDomain))
                            {
                                HtmlWeb webUtility = new HtmlWeb();
                                documnets.Add(webUtility.Load(HtmlLinkOld.Link));

                                if (documnets.Count > 6)
                                {
                                    break;
                                }
                            }

                        }

                        //Means, the domain could be main domain, or its diffrent sub domain in main domain.
                    }
                }

            }
            else
            {
                //todo: if the response is other than 200, we need to log it for review.

                //_logger.Error(
                //    string.Format(
                //        "Widget {0} failed to load. Server returned status code {1}. \r\nResponse Text - {2}",
                //        definition.Name, response.StatusCode, responseText));
            }


            List<HtmlLinkOld> linksForAnalysis = new List<HtmlLinkOld>();

            foreach (var item in documnets)
            {
                linksForAnalysis.AddRange(GetLinks(item,urlInfo));
            }

            var result = from l in linksForAnalysis
                         group l by l.Link into g
                         select new {Link = g.Key,Count = g.Count()
                         
                         ,Titles = (from lnk in linksForAnalysis
                                    where lnk.Link == g.Key
                                    select lnk.Title).ToList()
                         };



            var list = result.ToList();

            var orderedList = (from l in list
                               orderby l.Count descending
                                   select l).ToList();



        }


    }
}
